import json
import jsonpath
import scrapy
import time


class CtxwSpider(scrapy.Spider):
    name = 'ctxw'
    allowed_domains = ['chouti.com']
    count = 0
    lists = ['2020-11-29 10:40:30',
             '2020-11-30 10:40:30',
             '2020-12-01 10:40:30',
             '2020-12-02 10:40:30',
             ]
    url = 'https://dig.chouti.com/link/hot?afterTime=%s100000' % int(time.time())
    start_urls = [url]
    pl_url = 'https://dig.chouti.com/comments/show?id=%s'

    def parse(self, response):
        # print(response.text)
        jsons = json.loads(response.text)
        for i in jsons.get('data'):
            ids = i.get("id")
            title = i.get("title")
            original_img_url = i.get("original_img_url")
            nick = i.get("nick")
            ups = i.get("ups")
            if nick == None:
                nick = 0
            else:
                nick = i.get("nick")
            originalUrl = i.get("originalUrl")

            yield scrapy.Request(url=self.pl_url % ids, callback=self.parse_pl, cb_kwargs={
                'id': ids,
                'title': title,
                'nick': nick,
                'originalUrl': originalUrl,
                'ups': ups
            })
            # print(original_img_url)
            yield scrapy.Request(url=original_img_url, callback=self.parse_imgs, cb_kwargs={'ids': ids})
            print('end')
        self.count += 1
        if self.count == 4:
            return
        time_c = self.lists[self.count - 1]
        ts = int(time.mktime(time.strptime(time_c, "%Y-%m-%d %H:%M:%S")))
        url = 'https://dig.chouti.com/link/hot?afterTime=%s100000' % ts
        yield scrapy.Request(url=url, callback=self.parse)

    def parse_imgs(self, response, ids):
        print('1')
        yield {
            'type': 'img',
            'info': response.body,
            'title': str(ids) + '.jpg'
        }

    def parse_pl(self, response, id, title, nick, originalUrl, ups):
        jsons = json.loads(response.text)
        ner = jsonpath.jsonpath(jsons, '$..content')
        # print(ner)
        if ner:
            aa = set(ner)

            aa = list(aa)
        else:
            aa = '没有评论'
        item = {
            "type": "text",
            "id": id,
            "点赞": ups,
            "标题": title,
            "发布人": nick,
            "详情页url": originalUrl,
            "评论": aa,

        }

        yield item
